#!/usr/bin/python

import os
import sys
import random
import ConfigParser


config = ConfigParser.SafeConfigParser()
config.read(sys.argv[1])

featdir		= config.get('setting', 'featdir')
featspliter	= config.get('setting', 'featspliter')[1:-1]
featsize	= int(config.get('setting', 'featsize'))
realposstr	= config.get('setting', 'realposstr')
scorefile	= config.get('setting', 'scorefile')
outputfile	= config.get('setting', 'outputfile')
titleprefix	= config.get('setting', 'titleprefix')
scorespliter	= config.get('setting', 'scorespliter')[1:-1]
teamset		= config.get('setting', 'teamset')
classes		= int(config.get('setting', 'classes'))
randomsize	= int(config.get('setting', 'randomsize'))
attrifile	= config.get('setting', 'attrifile')
usetestset	= float(config.get('setting', 'usetestset'))
featextend	= config.get('setting', 'featextend')
randomseed	= int(config.get('setting', 'randomseed'))
skipfeatures	= config.get('setting', 'skipfeatures')
#scoreprefix	= config.get('setting', 'scoreprefix')

random.seed(randomseed)
# indicate real attribute
realpos = set()
realpossp = realposstr.split(':')
for i in range(0, len(realpossp)):
	realpos.add(int(realpossp[i].strip()))

# indicate skip attributes
skipatt = skipfeatures.strip().split(':')

# select teams to use
useteams = teamset.strip().split('_')
# read all attribute
attriset = {}
for i in range(1, featsize + 1):
	attriset[i] = set()

featdirs = featdir.split(':')

for singlefeatdir in featdirs:
	featurefilelist = os.listdir(singlefeatdir)
	for featurefile in featurefilelist:
		with open(singlefeatdir + '/' + featurefile, 'r') as ffeat:
			feats = ffeat.readlines()
			for feat in feats:
				attributes = feat.split(featspliter)
				for i in range(1, featsize + 1):
					attriset[i].add(attributes[i - 1].strip())

# read attribute name
attriname = {}
with open(attrifile, 'r') as fattri:
	attributes = fattri.readlines()
	for attri in attributes:
		line = attri.split(';')
		if len(line) == 2:
			attrisym = line[0].strip()
			index = int(line[1].strip())
			attriname[index] = attrisym

# read all score
scorelist = {}
scorefiles = scorefile.split(':')
titleprefixes = titleprefix.split(':')
for i in range(0, len(scorefiles)):
	singlescorefile = scorefiles[i]
	singletitleprefix = titleprefixes[i]
	with open(singlescorefile, 'r') as fscore:
		scorepos = set()
		lines = fscore.readlines()
		titles = lines[0].strip().split(scorespliter)
		for i in range(0, len(titles)):
			if titles[i].strip().startswith(singletitleprefix):
				scorepos.add(i)
		for i in range(1, len(lines)):
			line = lines[i].strip().split(scorespliter)
			for j in range(0, len(line)):
				if j in scorepos:
					triple = line[j].strip().split(':')
					if triple[0].strip() != '':
						#print triple[0]
						rawgroup = triple[0].split('/')[0]
						rawsenten = triple[0].split('/')[1]
						rawteam = triple[1]
						rawscore = triple[2].split('-')[0].strip()
						if rawscore != '' and rawteam in useteams:
							if not rawsenten in scorelist.keys():
								scorelist[rawsenten] = []
							scorelist[rawsenten].append(int(rawscore))

# zscore quantization the average score

avescore = {}
for senten in scorelist.keys():
        avescore[senten] = sum(scorelist[senten]) / float(len(scorelist[senten]))

tmpmaxscore = 0
tmpminscore = 1e+8
for senten in avescore.keys():
	if tmpmaxscore < avescore[senten]:
		tmpmaxscore = avescore[senten]
	if tmpminscore > avescore[senten]:
		tmpminscore = avescore[senten]
means = {}
meanset = {}
for i in range(0, classes):
	means[i] = (tmpmaxscore - tmpminscore) / (classes - 1) * i + tmpminscore

lastdis = 1e+8
currentdis = 1e+8 - 1
threshold = 1e-6
while lastdis - currentdis > threshold:
	for i in range(0, classes):
		meanset[i] = []
	for senten in avescore.keys():
		mindis = 1e+8
		index = -1
		for i in range(0, classes):
			dis = (avescore[senten] - means[i]) ** 2
			if mindis > dis:
				index = i
				mindis = dis
		meanset[index].append(senten)

	lastdis = currentdis
	currentdis = float(0)
	for i in range(0, classes):
		sumscore = float(0)
		for j in range(0, len(meanset[i])):
			sumscore += avescore[meanset[i][j]]
		if len(meanset[i]) != 0:
			means[i] = sumscore / len(meanset[i])
		for j in range(0, len(meanset[i])):
			currentdis += (avescore[meanset[i][j]] - means[i]) ** 2

print currentdis
print lastdis

finalscore = {}
for senten in avescore.keys():
	mindis = 1e+8
	index = -1
	for i in range(0, classes):
		dis = (avescore[senten] - means[i]) ** 2
		if mindis > dis:
			index = i 
			mindis = dis
	finalscore[senten] = index + 1
'''
avescore = {}
sumall = 0
print len(scorelist.keys())
for senten in scorelist.keys():
	avescore[senten] = sum(scorelist[senten]) / float(len(scorelist[senten]))
	sumall += avescore[senten]
aveall = sumall / len(avescore.keys())
sumvar = 0.0
for key in avescore.keys():
	sumvar += (avescore[key] - aveall) ** 2
stdvar = (sumvar / len(avescore.keys())) ** 0.5
finalscore = {}
for senten in avescore.keys():
	quanscore = (avescore[senten] - aveall) / stdvar
	if quanscore >= 1:
		finalscore[senten] = classes
	elif quanscore < -1:
		finalscore[senten] = 1
	else:
		finalscore[senten] = int((quanscore + 1) * 0.5 * (classes - 2) + 2)
'''
# select files with score
filewithscore = set([])
for singlefeatdir in featdirs:
	print singlefeatdir
	featurefilelist = os.listdir(singlefeatdir)
	for featurefile in featurefilelist:
		name = featurefile.split('.')[0]
		ext = featurefile.split('.')[1]
		if ext == featextend and name in avescore.keys():
			print featurefile
			filewithscore.add(name)

#print filewithscore
# writefile
def writearfffile(arfffilename, filelist):
#	print filelist
	attriindex = {}
	with open(arfffilename, 'w') as fout:
# generate header
		fout.write('@relation context_label\n')
		for i in range(1, featsize + 1):
			if str(i - 1) in skipatt:
				continue
			if not attriname[i] in attriindex:
				attris = ''
				for symble in attriset[i]:
					attris += symble + ','
				if i in realpos:
					fout.write('@attribute ' + attriname[i] + ' real\n')
				else:
					fout.write('@attribute ' + attriname[i] + ' {' + attris[0:-1].replace('%', '$') + '}\n')
				attriindex[attriname[i]] = 1
			else:
				attris = ''
				for symble in attriset[i]:
					attris += symble + ','
				if i in realpos:
					fout.write('@attribute ' + attriname[i] + str(attriindex[attriname[i]]) + ' real\n')
				else:
					fout.write('@attribute ' + attriname[i] + str(attriindex[attriname[i]]) + ' {' + attris[0:-1].replace('%', '$') + '}\n')
				attriindex[attriname[i]] += 1

		classset = ''
		for i in range(1, classes + 1):
			classset += str(i) + ','
		fout.write('@attribute class ' + '{' + classset[0:-1] + '}\n')
		fout.write('@data\n')

# read and generate all features
		for featdir in featdirs:
			for name in filelist:
				featfilename = featdir + '/' + name + '.' + featextend
				if os.path.exists(featfilename):
					print featfilename
					ffeat = open(featfilename, 'r')
					feats = ffeat.readlines()
					#print len(feats)
					samplefeats = random.sample(feats, randomsize)
					#samplefeats = random.sample(feats, len(feats))
					for feats in samplefeats:
						featall = feats.strip().replace('%', '$').split(' ')
						writeback = ''
						for j in range(0, len(featall)):
							if str(j) in skipatt:
								continue
							writeback += featall[j] + ','
						fout.write(writeback + str(finalscore[name]) + '\n')
	
# decide file mode
if usetestset != 0:
	trainset = random.sample(filewithscore, int(len(filewithscore) * usetestset))
	testset = []
	trainlist = ''
	testlist = ''
	for name in filewithscore:
		if not name in trainset:
			testset.append(name)
			testlist += '\t' + name + '\n'
		else:
			trainlist += '\t' + name + '\n'
	writearfffile(outputfile + '_Train.arff', trainset)
	print outputfile + '_Train.arff'
	writearfffile(outputfile + '_Test.arff', testset)
	print outputfile + '_Test.arff'
	with open(outputfile + '_Divide.log', 'w') as flog:
		flog.write('Train List:\n' + trainlist + 'Test List:\n' + testlist)
else:
	print filewithscore
	writearfffile(outputfile + '_Single.arff', filewithscore)
	print outputfile + '_Single.arff'
	with open(outputfile + '_Single.log', 'w') as flog:
		flog.write('List:\n')
		for name in filewithscore:
			flog.write('\t' + name +'\n')
